<!doctype html>



  


<html class="theme-next pisces use-motion" lang="zh-Hans">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>



<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />




  
  
  
  

  
    
    
  

  

  

  

  

  
    
    
    <link href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
  






<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.0" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="前端,nodejs,网页爬虫,async,bagpipe," />








  <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=5.1.0" />






<meta name="description" content="前言故事还得从头说起。乌云网挂掉之后,乌云知识库也无法访问了。曾经,在上面看到那么多优秀的安全类文章,一下子看不到了,颇觉得有点不适应。还好网上流传着民间的各种版本,于是我收集了一下,放在了Github上。这些文章只是一些html文件,并不包含页面上的图片。幸运的是,图片的域名static.wooyun.com还可以继续访问,因此有必要把这些图片也抓取下来。
Wooyun Drops 文章在线浏览">
<meta property="og:type" content="article">
<meta property="og:title" content="Nodejs大批量下载图片入坑指南(使用async和bagpipe处理大并发量请求)">
<meta property="og:url" content="http://jiji262.github.io/2016/08/06/nodejs-batch-download-images-with-async-and-bagpipe/index.html">
<meta property="og:site_name" content="令狐葱@前端笔记">
<meta property="og:description" content="前言故事还得从头说起。乌云网挂掉之后,乌云知识库也无法访问了。曾经,在上面看到那么多优秀的安全类文章,一下子看不到了,颇觉得有点不适应。还好网上流传着民间的各种版本,于是我收集了一下,放在了Github上。这些文章只是一些html文件,并不包含页面上的图片。幸运的是,图片的域名static.wooyun.com还可以继续访问,因此有必要把这些图片也抓取下来。
Wooyun Drops 文章在线浏览">
<meta property="og:updated_time" content="2017-02-22T01:16:55.000Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Nodejs大批量下载图片入坑指南(使用async和bagpipe处理大并发量请求)">
<meta name="twitter:description" content="前言故事还得从头说起。乌云网挂掉之后,乌云知识库也无法访问了。曾经,在上面看到那么多优秀的安全类文章,一下子看不到了,颇觉得有点不适应。还好网上流传着民间的各种版本,于是我收集了一下,放在了Github上。这些文章只是一些html文件,并不包含页面上的图片。幸运的是,图片的域名static.wooyun.com还可以继续访问,因此有必要把这些图片也抓取下来。
Wooyun Drops 文章在线浏览">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Pisces',
    sidebar: {"position":"post","display":"post"},
    fancybox: true,
    motion: true,
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="http://jiji262.github.io/2016/08/06/nodejs-batch-download-images-with-async-and-bagpipe/"/>





  <title> Nodejs大批量下载图片入坑指南(使用async和bagpipe处理大并发量请求) | 令狐葱@前端笔记 </title>
</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  





  <script type="text/javascript">
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?0fd825270d7f51597f963010828c65b8";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>










  
  
    
  

  <div class="container one-collumn sidebar-position-post page-post-detail ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-meta ">
  

  <div class="custom-logo-site-title">
    <a href="/"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">令狐葱@前端笔记</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
    
      <p class="site-subtitle"></p>
    
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="http://jiji262.github.io/2016/08/06/nodejs-batch-download-images-with-async-and-bagpipe/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="令狐葱">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="/images/avatar.gif">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="令狐葱@前端笔记">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="令狐葱@前端笔记" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                Nodejs大批量下载图片入坑指南(使用async和bagpipe处理大并发量请求)
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2016-08-06T20:10:00+08:00">
                2016-08-06
              </time>
            

            

            
          </span>

          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2016/08/06/nodejs-batch-download-images-with-async-and-bagpipe/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count ds-thread-count" data-thread-key="2016/08/06/nodejs-batch-download-images-with-async-and-bagpipe/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          
          

          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        <h1 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h1><p>故事还得从头说起。<a href="http://business.sohu.com/20160722/n460493827.shtml" target="_blank" rel="external">乌云网</a>挂掉之后,<a href="http://drops.wooyun.org/" target="_blank" rel="external">乌云知识库</a>也无法访问了。曾经,在上面看到那么多优秀的安全类文章,一下子看不到了,颇觉得有点不适应。还好网上流传着民间的各种版本,于是我收集了一下,放在了<a href="https://github.com/jiji262/wooyun_articles" target="_blank" rel="external">Github</a>上。这些文章只是一些html文件,并不包含页面上的图片。幸运的是,图片的域名static.wooyun.com还可以继续访问,因此有必要把这些图片也抓取下来。</p>
<h2 id="Wooyun-Drops-文章在线浏览"><a href="#Wooyun-Drops-文章在线浏览" class="headerlink" title="Wooyun Drops 文章在线浏览"></a>Wooyun Drops 文章在线浏览</h2><p><a href="https://jiji262.github.io/wooyun_articles/">Wooyun Drops 文章在线浏览</a><br><a href="https://github.com/jiji262/wooyun_articles" target="_blank" rel="external">Github: wooyun_articles</a></p>
<h1 id="使用Nodejs下载图片"><a href="#使用Nodejs下载图片" class="headerlink" title="使用Nodejs下载图片"></a>使用Nodejs下载图片</h1><p>抓取图片链接的过程在此不再详述,无非就是打开每个html页面,找到其中img标签的src属性。我们拿到了这些html页面的图片链接,是这个样子的:</p>
<figure class="highlight javascript"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">var</span> imageLinks = [</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20160315/2016031513484536696130.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20160315/2016031513484767273224.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20160315/2016031513485057874324.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20160315/2016031513485211060411.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20160315/201603151348542560759.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20160315/201603151348563741969.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20160315/201603151348582588879.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20160315/201603151349001461288.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20160315/201603151349032455899.png'</span>,</div><div class="line">    ......</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20150714/2015071404144944570.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20150714/2015071404144966345.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20150714/2015071404144915704.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20150714/2015071404144980399.png'</span>,</div><div class="line">    <span class="string">'http://static.wooyun.org//drops/20150714/2015071404144927633.png'</span></div><div class="line">];</div></pre></td></tr></table></figure>
<p>我们将其定义为一个模块备用(总共大约有13000个图片链接)。</p>
<p>在Nodejs中下载图片有很多解决方案,比如可以使用npm包<a href="https://github.com/kevva/download" target="_blank" rel="external">download</a>。本文中使用比较简单的版本,具体实现如下:<br><figure class="highlight javascript"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">var</span> fs = <span class="built_in">require</span>(<span class="string">"fs"</span>);</div><div class="line"><span class="keyword">var</span> path = <span class="built_in">require</span>(<span class="string">'path'</span>);</div><div class="line"><span class="keyword">var</span> request = <span class="built_in">require</span>(<span class="string">'request'</span>);</div><div class="line"></div><div class="line"><span class="keyword">var</span> downloadImage = <span class="function"><span class="keyword">function</span>(<span class="params">src, dest, callback</span>) </span>&#123;</div><div class="line">    request.head(src, <span class="function"><span class="keyword">function</span>(<span class="params">err, res, body</span>) </span>&#123;</div><div class="line">        <span class="comment">// console.log('content-type:', res.headers['content-type']);</span></div><div class="line">        <span class="comment">// console.log('content-length:', res.headers['content-length']);</span></div><div class="line">        <span class="keyword">if</span> (src) &#123;</div><div class="line">            request(src).pipe(fs.createWriteStream(dest)).on(<span class="string">'close'</span>, <span class="function"><span class="keyword">function</span>(<span class="params"></span>) </span>&#123;</div><div class="line">                callback(<span class="literal">null</span>, dest);</div><div class="line">            &#125;);</div><div class="line">        &#125;</div><div class="line">    &#125;);</div><div class="line">&#125;;</div></pre></td></tr></table></figure></p>
<p>代码来源: <a href="http://stackoverflow.com/questions/12740659/downloading-images-with-node-js" target="_blank" rel="external">stackoverflow</a>。</p>
<a id="more"></a>
<p>比如想要下载一个图片,可以这样子来做:<br><figure class="highlight javascript"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div></pre></td><td class="code"><pre><div class="line">downloadImage(<span class="string">"http://static.wooyun.org/20140918/2014091811544377515.png"</span>, <span class="string">"./1.png"</span>, <span class="function"><span class="keyword">function</span>(<span class="params">err, data</span>)</span>&#123;</div><div class="line">    <span class="keyword">if</span> (err) &#123;</div><div class="line">        <span class="built_in">console</span>.log(err)</div><div class="line">    &#125;</div><div class="line">    <span class="keyword">if</span> (data) &#123;</div><div class="line">        <span class="built_in">console</span>.log(<span class="string">"done: "</span> + data);</div><div class="line">    &#125;</div><div class="line">&#125;)</div></pre></td></tr></table></figure></p>
<p>对于我们想要的批量下载图片,最直观的做法就是写一个循环,然后去调用downloadImage方法。但是由于图片链接比较多,总是在下载到一部分图片后出现一些错误:<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div></pre></td><td class="code"><pre><div class="line">events.js:160</div><div class="line">      throw er; // Unhandled <span class="string">'error'</span> event</div><div class="line">      ^</div><div class="line"></div><div class="line">Error: Invalid URI <span class="string">"undefined"</span></div><div class="line">    at Request.init (/Users//dirread/node_modules/request/request.js:275:31)</div><div class="line">    at new Request (/Users/dirread/node_modules/request/request.js:129:8)</div><div class="line">    at request (/Users/dirread/node_modules/request/index.js:55:10)</div><div class="line">    at Request._callback (/Users//down.js:21:11)</div><div class="line">    at self.callback (/Users//dirread/node_modules/request/request.js:187:22)</div><div class="line">    at emitOne (events.js:96:13)</div><div class="line">    at Request.emit (events.js:188:7)</div></pre></td></tr></table></figure></p>
<p>因此只有考虑使用异步方式来处理。</p>
<h1 id="解决方案一-使用async异步批量下载图片"><a href="#解决方案一-使用async异步批量下载图片" class="headerlink" title="解决方案一: 使用async异步批量下载图片"></a>解决方案一: 使用async异步批量下载图片</h1><p><a href="https://github.com/caolan/async" target="_blank" rel="external">Async</a>是一个流程控制工具包，提供了直接而强大的异步功能。其提供了大约20个函数，包括常用的 map, reduce, filter, forEach 等，异步流程控制模式包括，串行(series)，并行(parallel)，瀑布(waterfall)等。</p>
<h2 id="安装async"><a href="#安装async" class="headerlink" title="安装async"></a>安装async</h2><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">npm install async --save</div></pre></td></tr></table></figure>
<p>其使用方式也比较简单,比如下面的示例:</p>
<figure class="highlight javascript"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">async</span>.map([<span class="string">'file1'</span>,<span class="string">'file2'</span>,<span class="string">'file3'</span>], fs.stat, <span class="function"><span class="keyword">function</span>(<span class="params">err, results</span>)</span>&#123;</div><div class="line">    <span class="comment">// results is now an array of stats for each file</span></div><div class="line">&#125;);</div><div class="line"></div><div class="line"><span class="keyword">async</span>.filter([<span class="string">'file1'</span>,<span class="string">'file2'</span>,<span class="string">'file3'</span>], <span class="function"><span class="keyword">function</span>(<span class="params">filePath, callback</span>) </span>&#123;</div><div class="line">  fs.access(filePath, <span class="function"><span class="keyword">function</span>(<span class="params">err</span>) </span>&#123;</div><div class="line">    callback(<span class="literal">null</span>, !err)</div><div class="line">  &#125;);</div><div class="line">&#125;, <span class="function"><span class="keyword">function</span>(<span class="params">err, results</span>)</span>&#123;</div><div class="line">    <span class="comment">// results now equals an array of the existing files</span></div><div class="line">&#125;);</div><div class="line"></div><div class="line"><span class="keyword">async</span>.parallel([</div><div class="line">    <span class="function"><span class="keyword">function</span>(<span class="params">callback</span>)</span>&#123; ... &#125;,</div><div class="line">    <span class="function"><span class="keyword">function</span>(<span class="params">callback</span>)</span>&#123; ... &#125;</div><div class="line">], <span class="function"><span class="keyword">function</span>(<span class="params">err, results</span>) </span>&#123;</div><div class="line">    <span class="comment">// optional callback</span></div><div class="line">&#125;;</div><div class="line"></div><div class="line"><span class="keyword">async</span>.series([</div><div class="line">    <span class="function"><span class="keyword">function</span>(<span class="params">callback</span>)</span>&#123; ... &#125;,</div><div class="line">    <span class="function"><span class="keyword">function</span>(<span class="params">callback</span>)</span>&#123; ... &#125;</div><div class="line">]);</div></pre></td></tr></table></figure>
<p>关于async的示例在此不再展开,具体可以参考<a href="https://github.com/alsotang/async_demo" target="_blank" rel="external">alsotang的demo和教程</a>。</p>
<h2 id="使用async下载图片"><a href="#使用async下载图片" class="headerlink" title="使用async下载图片"></a>使用async下载图片</h2><p>回到我们的程序,我们可以使用来实现。具体代码如下:</p>
<figure class="highlight javascript"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">async</span>.mapSeries(imageLinks, <span class="function"><span class="keyword">function</span>(<span class="params">item, callback</span>) </span>&#123;</div><div class="line">    <span class="comment">//console.log(item);</span></div><div class="line">    setTimeout(<span class="function"><span class="keyword">function</span>(<span class="params"></span>) </span>&#123;</div><div class="line">        <span class="keyword">if</span> (item.indexOf(<span class="string">"http://static.wooyun.org"</span>) === <span class="number">0</span>) &#123;</div><div class="line">            <span class="keyword">var</span> destImage = path.resolve(<span class="string">"./images/"</span>, item.split(<span class="string">"/"</span>)[item.split(<span class="string">"/"</span>).length <span class="number">-1</span>]);</div><div class="line">            downloadImage(item, destImage, <span class="function"><span class="keyword">function</span>(<span class="params">err, data</span>)</span>&#123;</div><div class="line">                <span class="built_in">console</span>.log(<span class="string">"["</span>+ index++ +<span class="string">"]: "</span> + data);</div><div class="line">            &#125;);</div><div class="line">            </div><div class="line">        &#125;</div><div class="line">        callback(<span class="literal">null</span>, item);</div><div class="line">    &#125;, <span class="number">100</span>);</div><div class="line"></div><div class="line"></div><div class="line">&#125;, <span class="function"><span class="keyword">function</span>(<span class="params">err, results</span>) </span>&#123;&#125;);</div></pre></td></tr></table></figure>
<p>完整代码可以参看<a href="https://github.com/jiji262/wooyun_articles/blob/master/image_download/async_version/down.js" target="_blank" rel="external">github上</a>。</p>
<h2 id="运行"><a href="#运行" class="headerlink" title="运行"></a>运行</h2><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">node down.js</div></pre></td></tr></table></figure>
<p>即可看到结果。</p>
<h3 id="踩坑提醒"><a href="#踩坑提醒" class="headerlink" title="踩坑提醒"></a>踩坑提醒</h3><p>在Mac上使用的时候,偶尔会出现这样的错误:<br><figure class="highlight gauss"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line"><span class="built_in">Error</span>: EMFILE, too many <span class="keyword">open</span> files</div></pre></td></tr></table></figure></p>
<p>有可能是因为Mac对并发打开的文件数限制的比较小,一般为256.所以需要修改一下。<br>在命令行执行:<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line"><span class="built_in">echo</span> kern.maxfiles=65536 | sudo tee <span class="_">-a</span> /etc/sysctl.conf</div><div class="line"><span class="built_in">echo</span> kern.maxfilesperproc=65536 | sudo tee <span class="_">-a</span> /etc/sysctl.conf</div><div class="line">sudo sysctl -w kern.maxfiles=65536</div><div class="line">sudo sysctl -w kern.maxfilesperproc=65536</div><div class="line"><span class="built_in">ulimit</span> -n 65536 65536</div></pre></td></tr></table></figure></p>
<p>然后可以在你的.bashrc文件中加入一行:<br><figure class="highlight lsl"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">ulimit -n <span class="number">65536</span> <span class="number">65536</span></div></pre></td></tr></table></figure></p>
<p>解决方案来自<a href="http://stackoverflow.com/questions/19981065/nodejs-error-emfile-too-many-open-files-on-mac-os" target="_blank" rel="external">stackoverflow</a>.</p>
<h1 id="解决方案二-使用bagpipe批量下载图片"><a href="#解决方案二-使用bagpipe批量下载图片" class="headerlink" title="解决方案二: 使用bagpipe批量下载图片"></a>解决方案二: 使用bagpipe批量下载图片</h1><p><a href="https://github.com/JacksonTian/bagpipe/" target="_blank" rel="external">bagpile</a>是朴灵大大做的一个在nodejs中控制并发执行的模块。</p>
<h2 id="安装bagpipe"><a href="#安装bagpipe" class="headerlink" title="安装bagpipe"></a>安装bagpipe</h2><p>其安装和使用也比较简单:<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">npm install bagpipe --save</div></pre></td></tr></table></figure></p>
<p>使用示例:<br><figure class="highlight javascript"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">var</span> bagpipe = <span class="keyword">new</span> Bagpipe(<span class="number">10</span>);</div><div class="line"></div><div class="line"><span class="keyword">var</span> files = [<span class="string">'这里有很多很多文件'</span>];</div><div class="line"><span class="keyword">for</span> (<span class="keyword">var</span> i = <span class="number">0</span>; i &lt; files.length; i++) &#123;</div><div class="line">  <span class="comment">// fs.readFile(files[i], 'utf-8', function (err, data) &#123;</span></div><div class="line">  bagpipe.push(fs.readFile, files[i], <span class="string">'utf-8'</span>, <span class="function"><span class="keyword">function</span> (<span class="params">err, data</span>) </span>&#123;</div><div class="line">    <span class="comment">// 不会因为文件描述符过多出错</span></div><div class="line">    <span class="comment">// 妥妥的</span></div><div class="line">  &#125;);</div><div class="line">&#125;</div></pre></td></tr></table></figure></p>
<h2 id="使用bagpipe批量下载图片"><a href="#使用bagpipe批量下载图片" class="headerlink" title="使用bagpipe批量下载图片"></a>使用bagpipe批量下载图片</h2><p>将我们的代码稍做修改,就可以使用bagpipe了。具体代码如下:<br><figure class="highlight javascript"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">for</span> (<span class="keyword">var</span> i = <span class="number">0</span>; i &lt; imageLinks.length; i++) &#123;</div><div class="line">    <span class="keyword">if</span> (imageLinks[i].indexOf(<span class="string">"http://static.wooyun.org"</span>) === <span class="number">0</span>) &#123;</div><div class="line">        <span class="keyword">var</span> destImage = path.resolve(<span class="string">"./images/"</span>, imageLinks[i].split(<span class="string">"/"</span>)[imageLinks[i].split(<span class="string">"/"</span>).length <span class="number">-1</span>]);</div><div class="line">        bagpipe.push(downloadImage, imageLinks[i], destImage, <span class="function"><span class="keyword">function</span>(<span class="params">err, data</span>) </span>&#123;</div><div class="line">            <span class="built_in">console</span>.log(<span class="string">"["</span>+ index++ +<span class="string">"]: "</span> + data);</div><div class="line">        &#125;);</div><div class="line">    &#125;</div><div class="line">&#125;</div></pre></td></tr></table></figure></p>
<p>完整代码可以参考<a href="https://github.com/jiji262/wooyun_articles/blob/master/image_download/bagpipe_version/down.js" target="_blank" rel="external">github上</a>。</p>
<h2 id="运行-1"><a href="#运行-1" class="headerlink" title="运行"></a>运行</h2><figure class="highlight crmsh"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">node</span> <span class="title">down</span>.js</div></pre></td></tr></table></figure>
<p>即可看到结果。</p>
<h1 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h1><p>async和bagpipe都是很优秀的nodejs包,本身async功能十分强大,bagpipe使用起来简单方便,对原有的异步代码几乎不必做太多改动。因此可以根据自己喜好选择使用。</p>

      
    </div>

    <div>
      
        
<div id="wechat_subscriber" style="display: block; padding: 10px 0; margin: 20px auto; width: 100%; text-align: center">
    <img id="wechat_subscriber_qcode" src="/uploads/wechatmp.jpg" alt="令狐葱 wechat" style="width: 200px; max-width: 100%;"/>
    <div>欢迎您扫一扫上面的二维码，订阅我的微信公众号！</div>
</div>


      
    </div>

    <div>
      
        
  <div style="padding: 10px 0; margin: 20px auto; width: 90%; text-align: center;">
    <div></div>
    <button id="rewardButton" disable="enable" onclick="var qr = document.getElementById('QR'); if (qr.style.display === 'none') {qr.style.display='block';} else {qr.style.display='none'}">
      <span>赏</span>
    </button>
    <div id="QR" style="display: none;">
      
        <div id="wechat" style="display: inline-block">
          <img id="wechat_qr" src="/uploads/wechatpay.png" alt="令狐葱 WeChat Pay"/>
          <p>微信打赏</p>
        </div>
      
      
        <div id="alipay" style="display: inline-block">
          <img id="alipay_qr" src="/uploads/alipaypay.png" alt="令狐葱 Alipay"/>
          <p>支付宝打赏</p>
        </div>
      
    </div>
  </div>


      
    </div>


    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/前端/" rel="tag"># 前端</a>
          
            <a href="/tags/nodejs/" rel="tag"># nodejs</a>
          
            <a href="/tags/网页爬虫/" rel="tag"># 网页爬虫</a>
          
            <a href="/tags/async/" rel="tag"># async</a>
          
            <a href="/tags/bagpipe/" rel="tag"># bagpipe</a>
          
        </div>
      

      
        
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2016/07/26/react-testing-with-karma-and-webpack-and-coverage-report/" rel="next" title="手把手教你为React Boilerplate添加测试代码覆盖率报告">
                <i class="fa fa-chevron-left"></i> 手把手教你为React Boilerplate添加测试代码覆盖率报告
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2016/08/18/ubuntu_1604_web_server_config_LEMP/" rel="prev" title="手把手教你从头搭建Web服务器及LEMP环境(使用Ubuntu16.04+Nginx+MySQL+PHP)">
                手把手教你从头搭建Web服务器及LEMP环境(使用Ubuntu16.04+Nginx+MySQL+PHP) <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </article>



    <div class="post-spread">
      
        <!-- JiaThis Button BEGIN -->
<div class="jiathis_style">
  <a class="jiathis_button_tsina"></a>
  <a class="jiathis_button_tqq"></a>
  <a class="jiathis_button_weixin"></a>
  <a class="jiathis_button_cqq"></a>
  <a class="jiathis_button_douban"></a>
  <a class="jiathis_button_renren"></a>
  <a class="jiathis_button_qzone"></a>
  <a class="jiathis_button_kaixin001"></a>
  <a class="jiathis_button_copy"></a>
  <a href="http://www.jiathis.com/share" class="jiathis jiathis_txt jiathis_separator jtico jtico_jiathis" target="_blank"></a>
  <a class="jiathis_counter_style"></a>
</div>
<script type="text/javascript" >
  var jiathis_config={
    hideMore:false
  }
</script>
<script type="text/javascript" src="http://v3.jiathis.com/code/jia.js" charset="utf-8"></script>
<!-- JiaThis Button END -->

      
    </div>
  </div>

          
          </div>
          


          
  <div class="comments" id="comments">
    
      <div class="ds-thread" data-thread-key="2016/08/06/nodejs-batch-download-images-with-async-and-bagpipe/"
           data-title="Nodejs大批量下载图片入坑指南(使用async和bagpipe处理大并发量请求)" data-url="http://jiji262.github.io/2016/08/06/nodejs-batch-download-images-with-async-and-bagpipe/">
      </div>
    
  </div>


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap" >
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="/images/avatar.gif"
               alt="令狐葱" />
          <p class="site-author-name" itemprop="name">令狐葱</p>
           
              <p class="site-description motion-element" itemprop="description">去留无意，闲看庭前花开花落；宠辱不惊，漫随天外云卷云舒。</p>
          
        </div>
        <nav class="site-state motion-element">
        
          
            <div class="site-state-item site-state-posts">
              <a href="/">
                <span class="site-state-item-count">9</span>
                <span class="site-state-item-name">日志</span>
              </a>
            </div>
          

          

          
            <div class="site-state-item site-state-tags">
              <a href="/tags">
                <span class="site-state-item-count">28</span>
                <span class="site-state-item-name">标签</span>
              </a>
            </div>
          

        </nav>

        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="https://github.com/jiji262" target="_blank" title="GitHub">
                  
                    <i class="fa fa-fw fa-github"></i>
                  
                  GitHub
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="https://twitter.com/linghucong" target="_blank" title="Twitter">
                  
                    <i class="fa fa-fw fa-twitter"></i>
                  
                  Twitter
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://weibo.com/jiji262" target="_blank" title="微博">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  微博
                </a>
              </span>
            
          
        </div>

        
        

        
        

        


      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#前言"><span class="nav-number">1.</span> <span class="nav-text">前言</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#Wooyun-Drops-文章在线浏览"><span class="nav-number">1.1.</span> <span class="nav-text">Wooyun Drops 文章在线浏览</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#使用Nodejs下载图片"><span class="nav-number">2.</span> <span class="nav-text">使用Nodejs下载图片</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#解决方案一-使用async异步批量下载图片"><span class="nav-number">3.</span> <span class="nav-text">解决方案一: 使用async异步批量下载图片</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#安装async"><span class="nav-number">3.1.</span> <span class="nav-text">安装async</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#使用async下载图片"><span class="nav-number">3.2.</span> <span class="nav-text">使用async下载图片</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#运行"><span class="nav-number">3.3.</span> <span class="nav-text">运行</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#踩坑提醒"><span class="nav-number">3.3.1.</span> <span class="nav-text">踩坑提醒</span></a></li></ol></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#解决方案二-使用bagpipe批量下载图片"><span class="nav-number">4.</span> <span class="nav-text">解决方案二: 使用bagpipe批量下载图片</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#安装bagpipe"><span class="nav-number">4.1.</span> <span class="nav-text">安装bagpipe</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#使用bagpipe批量下载图片"><span class="nav-number">4.2.</span> <span class="nav-text">使用bagpipe批量下载图片</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#运行-1"><span class="nav-number">4.3.</span> <span class="nav-text">运行</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#结语"><span class="nav-number">5.</span> <span class="nav-text">结语</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  
  &copy;  2015 - 
  <span itemprop="copyrightYear">2017</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">令狐葱</span>
</div>


<div class="powered-by">
  由 <a class="theme-link" href="https://hexo.io">Hexo</a> 强力驱动
</div>

<div class="theme-info">
  主题 -
  <a class="theme-link" href="https://github.com/iissnan/hexo-theme-next">
    NexT.Pisces
  </a>
</div>


        

        
      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
    </div>
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  




  
  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.0"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.0"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=5.1.0"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.1.0"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.0"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.0"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.0"></script>



  

  
    
  

  <script type="text/javascript">
    var duoshuoQuery = {short_name:"jiji262"};
    (function() {
      var ds = document.createElement('script');
      ds.type = 'text/javascript';ds.async = true;
      ds.id = 'duoshuo-script';
      ds.src = (document.location.protocol == 'https:' ? 'https:' : 'http:') + '//static.duoshuo.com/embed.js';
      ds.charset = 'UTF-8';
      (document.getElementsByTagName('head')[0]
      || document.getElementsByTagName('body')[0]).appendChild(ds);
    })();
  </script>

  
    
    
    <script src="/lib/ua-parser-js/dist/ua-parser.min.js?v=0.7.9"></script>
    <script src="/js/src/hook-duoshuo.js"></script>
  













  
  

  

  

  

  


</body>
</html>
